#!/usr/bin/env python3
from do_get_aminer_data import *
from config import *
import pymongo
import pandas as pd
from multiprocessing import Pool
import json
import codecs
import os

#配置mongodb的信息
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_RESEARCHER_SEX_DB]

'''
读取paper数据库中的数据
'''
def get_db_data(university_name):
    data = db[university_name].find({}).batch_size(1)
    return data


'''
将collection保存txt文件中
'''
def save_collection_as_txt(university_name):
    data = get_db_data(university_name)
    i = 0
    try:
        with open("./univeristy_sex_info_list/" + university_name + ".txt",mode = 'a+',encoding = 'utf-8') as f:
            for data_item in data:
                f.write(str({"AuN":data_item.get("AuN"),"AfN":data_item.get("AfN")}) + '\n')
        f.close()
    except FileNotFoundError:
        os.mknod(university_name + ".txt")
        print(2222)

'''
读取university_sex_info_list中的数据然后多线程调用性别预测接口
'''
def use_university_data_get_sex_info(university_name):
    #按行读取对应univeristy_name.txt中的数据
    with open("./after_delete_repeat_univeristy_sex_info_list/" + university_name + ".txt",mode = 'r+',encoding = 'utf-8') as f:
        content = f.read().splitlines()
    f.close()
    content_copy = content
    i = 0
    for data_item in content:
        print("到了"+ university_name + "第" + str(i) +"位了")
        data = eval(data_item)

        # print(data.get("AuN"))
        status = do_use_university_data_get_sex_info(data,university_name,i)
        i += 1
        if status == 200:
            #如果返回数据成功且成功修改collection中的数据，则将该条数据从content中删除
            #只能成功一条删除一条
            # with open("./after_delete_repeat_univeristy_sex_info_list/" + university_name + ".txt",mode = 'a+',encoding = 'utf-8') as f:
            #     content = f.readlines()
            content_copy.remove(data_item)
        elif status == 40303:
            #将content中的数据重新存入txt
            #先将文件中的数据清空后再添加数据
            os.remove("./after_delete_repeat_univeristy_sex_info_list/" + university_name + ".txt")
            with open("./after_delete_repeat_univeristy_sex_info_list/" + university_name + ".txt",mode = 'a+',encoding = 'utf-8') as f:
                for data_item in content_copy:
                    f.write(data_item + '\n')
            f.close()
            exit()
            # pass
        else:
            pass

    os.remove("./after_delete_repeat_univeristy_sex_info_list/" + university_name + ".txt")
    with open("./after_delete_repeat_univeristy_sex_info_list/" + university_name + ".txt",mode = 'a+',encoding = 'utf-8') as f:
        for data_item in content_copy:
            f.write(data_item + '\n')
    f.close()

'''
实现上面的方法
'''
def do_use_university_data_get_sex_info(data,university_name,i):
    researcher_affiliation = data.get("AfN")
    researcher_name = data.get("AuN")
    # print(researcher_affiliation)
    # 多线程去调 放在线程池
    get_aminer_data = do_get_aminer_data(researcher_name,researcher_affiliation)
    aminer_data_json,status = get_aminer_data.do_connect()
    # print("aminer_data_json:",aminer_data_json)
    if status == 200:
        gender_info = aminer_data_json.get("Final")
        print(gender_info)
        if gender_info.get("gender") == 'male':
            return status
        else:
            result = db[university_name].update({"AuN":researcher_name,"AfN":researcher_affiliation},{"$set":{"Sex":0}})
            if result:
                return status
            else:
                print("修改"+ university_name +"的第"+ str(i) +"条数据出问题!!!!!!!!!!!")
                return 40303
    elif status == 40303:
        print("服务器无法响应"+ university_name +"的第"+ str(i)+ "条数据-------------")
        return status
    else:
        print("获取" + university_name + "的第" + str(i) + "条数据失败~~~~~~~~~~~~")
        print(status)
        return status

'''
消除重复researcher_info数据
'''
def delete_repeat_researcher_info(university_name):
    with open("./univeristy_sex_info_list/" + university_name + ".txt",mode = 'r',encoding = 'utf-8') as f:
        content = f.read().splitlines()
        # print(content)
    f.close()
    #去重操作
    after_delete_repeat_data = list(set(content))
    with open("./after_delete_repeat_univeristy_sex_info_list/" + university_name + ".txt",mode = 'a+',encoding = 'utf-8') as ff:
        for data_item in after_delete_repeat_data:
            ff.write(data_item + '\n')
    ff.close()



'''
主函数
'''
def main():
    pool = Pool()
    #将collection中的数据保存到txt中
    # pool.map(save_collection_as_txt,UNIVERSITY_LIST2)

    #将txt中的数据去重
    # pool.map(delete_repeat_researcher_info,UNIVERSITY_LIST)

    #将去重后的数据多线程调用性别预测接口
    pool.map(use_university_data_get_sex_info,UNIVERSITY_LIST)



if __name__ == '__main__':
    main()
